#!/usr/bin/env python

from honeyclient import *
import getopt, md5, popen2, sys, time

import elementtree.ElementTree as ET

hc = HttpHoneyClient()
hc.ignore_domain('.google.com')
hc.maxdepth = 4
hc.cnt = 0
hc.digraph = []
root = ET.Element("html")

def usage(progname):
    usagestr = """%s [OPTIONS] url
HoneyClient spider tool

OPTIONS:
    -o file     dump the XML output to the named file
    -1          only analyze the one URL
""" % progname
    print usagestr
    sys.exit(1)

def scan(body):
    c = {'ClamAV': clamscan(body)}
    return c

def clamscan(body):
    m = md5.md5(body).hexdigest()
    f = open('/tmp/hc-walk-scan-%s' % m, 'wb')
    f.write(body)
    f.close()
    if len(body) == 0: return '-'
    try: os.stat('/tmp/clamd')
    except OSError: return m, '!!!'
    o, i, e = popen2.popen3('/usr/local/bin/clamdscan /tmp/hc-walk-scan-%s' % m)
    try: r = o.read().split('\n')[0].split(':')[1].strip()
    except IndexError: r = '???'
    o.close()
    i.close()
    e.close()
    return m, r

def walk(hc, URL, referrer, visited, depth, options):
    if referrer and not depth.has_key(URL): 
        depth[URL] = depth[referrer] + 1
    if depth[URL] > hc.maxdepth: return
    hc.cnt += 1
    if referrer: hc.digraph.append('\t"%s" -> "%s";' % (referrer, URL))
    else: hc.digraph.append('\t%s [shape=box, color=lightblue, style=filled];' % URL)

    url_visited = ET.SubElement(options['urltree'], 'url_visited')
    url_visited.set("url", URL)
    url_visited.set('timestamp', time.asctime(time.gmtime()))
    if URL.lower().startswith('http:') or URL.lower().startswith('https'):
        url_visited.set('style', 'absolute')
    else: url_visited.set('style', 'relative')
    url_visited.set('referrer', referrer or 'None')
    visited.append(URL)
    try: res = hc.get(URL, referrer=referrer)
    except Exception, e: return
    js = hc.js_eval(URL)
    repeat = True
    newjs = js
    run_cnt = 0
    while repeat:
        if 'document.write' in newjs[0].lower() or '<script' in newjs[0].lower():
            if run_cnt < 10:
                page = Page(newjs[0], URL)
                newjs = hc.js_eval(url=URL, html=page.contents)
                run_cnt += 1
                js += newjs
                hc.pages[URL].links += page.links
        else:
            repeat = False
            break

    content = ET.SubElement(url_visited, "content")
    av_scan = ET.SubElement(content, "antivirus")
    for k, v in scan(res).iteritems():
        engine = ET.SubElement(av_scan, "engine")
        engine.set('av_engine', k)
        engine.set('md5', v[0])
        engine.set('content-length', '%d' % len(res))
        try: engine.set('result', v[1])
        except IndexError: engine.set('result', 'error') 

    outlinks = ET.SubElement(url_visited, 'links_outgoing')
    for l in hc.pages[URL].links:
        outlink = ET.SubElement(outlinks, 'link')
        if l.lower().startswith('http:') or l.lower().startswith('https'):
            outlink.set('style', 'absolute')
        else: outlink.set('style', 'relative')
        outlink.set('url', l)

    if options['do_one']: return
    for url in hc.pages[URL].links:
        if url not in visited and url.startswith('http'):
            host = url.split('/')[2]
            if not hc.ignored(host):
                walk(hc, url, URL, visited, depth, options)

def main(argv):
    global root
    
    try: opts, args = getopt.getopt(argv[1:], 'dD1ho:')
    except: usage(sys.argv[0])

    options = {}
    options['do_one'] = False
    options['output'] = None
    for o, a in opts:
        if   o == '-1': options['do_one'] = True
        elif o == '-o': options['output'] = a
        else: usage(sys.argv[0])
    url = args[0]

    start = time.time()
    arguments = ET.SubElement(root, "arguments")
    for o, a in opts:
        arguments.set(o.replace('-', ''), a)
    arguments.set('argsleft', ' '.join(args))
    times = ET.SubElement(root, "runtime")
    times.set("start", time.asctime(time.gmtime()))
    urltree = ET.SubElement(root, "urls_visited")
    options['urltree'] = urltree

    try: walk(hc, url, False, [], {url: 0}, options)
    except KeyboardInterrupt: pass # print ''

    end = time.time()
    times.set("end", time.asctime(time.gmtime()))
    times.set("runtime", '%.2f' % (end-start))

    digraph = ET.SubElement(root, "digraph")
    digraph.text = '[CDATA[digraph %s {\n%s\n}]]' % (url, '\n'.join(hc.digraph))
    if options['output'] is None: ET.dump(root)
    else: open(options['output'], 'w').write(ET.tostring(root))

if __name__ == '__main__':
    main(sys.argv)
